__author__ = 'vincent Liu'

import pandas as pd
import numpy as np
import time
import os
import sys
import re
import downloaddpx

xt_dpx_path = 'z:\\data\\WindDB\\dpx\\'
dfw_dpx_path = 'z:\\data\\DFW\\dpx\\'
output_path = 'z:\\data\\DFW\\dpx\\dpxCheck\\'


def load_xt_dpx(curdate):
    fname = xt_dpx_path + curdate + '.dpx.csv'
    data = pd.read_csv(fname, encoding="gb18030", usecols=['#TICKER', 'OPEN', 'HIGH', 'LOW', 'CLOSE', 'VOLUME', 'VWAP'])
    data['VWAP'] = data['VWAP'].round(2)
    (data[data['VOLUME'] == 0])['CLOSE'] = 0
    (data[data['VOLUME'] == 0])['OPEN'] = 0
    (data[data['VOLUME'] == 0])['HIGH'] = 0
    (data[data['VOLUME'] == 0])['LOW'] = 0
    (data[data['VOLUME'] == 0])['VWAP'] = 0
    data['#TICKER'] = data['#TICKER'].apply(lambda x: '{0:>06}'.format(x))
    data.set_index('#TICKER', drop=True, inplace=True)
    data.sort(inplace=True)
    return data


def load_dfw_onefile(fname, market_flag):
    data = pd.read_csv(fname, header=None, encoding="gb18030", usecols=[1, 3, 4, 5, 6, 7, 8], skiprows=1,
                       names=['#TICKER', 'OPEN', 'HIGH', 'LOW', 'CLOSE', 'VOLUME', 'VWAP'])
    data['#TICKER'] = data['#TICKER'].apply(lambda x: '{0:>06}'.format(x))
    if market_flag == 'sh':
        qualify_tk = data['#TICKER'].str[0:3].isin(['600', '601', '603'])
    else:
        qualify_tk = data['#TICKER'].str[0:3].isin(['000', '300', '002'])
    data = data[qualify_tk]
    return data


def load_dfw_dpx(curdate):
    sh_file = dfw_dpx_path + "tb_day_sh_" + curdate + ".csv"
    sz_file = dfw_dpx_path + "tb_day_sz_" + curdate + ".csv"
    sh_data = load_dfw_onefile(sh_file, 'sh')
    sz_data = load_dfw_onefile(sz_file, 'sz')
    data = pd.concat([sh_data, sz_data])
    data['VOLUME'] = data['VOLUME'] * 100
    data['VWAP'] = (data['VWAP'] / (data['VOLUME'])).round(2)
    data = data[data['VWAP'] > 0]
    data.set_index('#TICKER', drop=True, inplace=True)
    data.sort(inplace=True)
    return data


def compare_type(col_name, interA, interB, cmp_size, reason_str):
    idx = (abs(interA[col_name] - interB[col_name]) > cmp_size)
    df = pd.concat([interA[idx][col_name], interB[idx][col_name],
                    abs(interA[idx][col_name] - interB[idx][col_name])], axis=1)
    df.columns = ['XT', 'DFW', 'DIFF']
    df['REASON'] = reason_str
    return df


def dpx_check_bydate():
    ftpret = downloaddpx.Download_dfw_dpx()
    if (ftpret == -1):
        return -1
    curdate = time.strftime("%Y%m%d")
    B = load_dfw_dpx(curdate)
    A = load_xt_dpx(curdate)
    setA = set(A.index.tolist())
    setB = set(B.index.tolist())
    inter_idx = setA.intersection(setB)
    InBoth = setA | setB
    OnlyInDFW = setB.difference(setA)
    OnlyInXT = setA.difference(setB)

    opt_list = list()
    # only exists in XT
    if len(OnlyInXT) > 0:
        ab_list = list()
        for k in OnlyInXT:
            ab_list.append({'#TICKER': k, 'XT': 1, 'DFW': 0, 'DIFF': 1, 'REASON': 'OnlyInXT'})
        opt_list.append(pd.DataFrame(ab_list))

    # only exists in DFW
    if len(OnlyInDFW) > 0:
        ab_list = list()
        for k in OnlyInDFW:
            ab_list.append({'#TICKER': k, 'XT': 0, 'DFW': 1, 'DIFF': 1, 'REASON': 'OnlyInDFW'})
            opt_list.append(pd.DataFrame(ab_list))
    opt_df = pd.concat(opt_list)[['#TICKER', 'XT', 'DFW', 'DIFF', 'REASON']]
    opt_df.set_index('#TICKER', inplace=True)
    interA = A.loc[inter_idx]
    interB = B.loc[inter_idx]
    open_df = compare_type('OPEN', interA, interB, 0.01, 'OPEN_ERR')
    high_df = compare_type('HIGH', interA, interB, 0.01, 'HIGH_ERR')
    low_df = compare_type('LOW', interA, interB, 0.01, 'LOW_ERR')
    close_df = compare_type('CLOSE', interA, interB, 0.01, 'CLOSE_ERR')
    volume_df = compare_type('VOLUME', interA, interB, 100, 'VOLUME_ERR')
    #vwap1_df = compare_type('VWAP',interA,interB,0.01)
    vwap2_df = compare_type('VWAP', interA, interB, 0.02, 'VWAP2Cents')
    vwap5_df = compare_type('VWAP', interA, interB, 0.05, 'VWAP5Cents')
    result_df = pd.concat([open_df, high_df, low_df, close_df, volume_df, vwap2_df, vwap5_df, opt_df])
    result_df.sort(inplace=True)
    result_df.to_csv(output_path + curdate + 'dpxCheckDetail.csv')

    opt_dict = {
        'Date': curdate,
        'InBoth': len(InBoth),
        'OnlyInXT': len(OnlyInXT),
        'OnlyInDFW': len(OnlyInDFW),
        'OpenErr': len(open_df.index),
        'CloseErr': len(close_df.index),
        'HighErr': len(high_df.index),
        'LowErr': len(low_df.index),
        'VolumeErr': len(volume_df.index),
        'Vwap2CentsErr': len(vwap2_df.index),
        'Vwap5CentsErr': len(vwap5_df.index),
    }

    summary_header = ['Date',
                      'InBoth',
                      'OnlyInDFW',
                      'OnlyInXT',
                      'OpenErr',
                      'CloseErr',
                      'HighErr',
                      'LowErr',
                      'VolumeErr',
                      'Vwap2CentsErr',
                      'Vwap5CentsErr',
                      ]
    pd.DataFrame([opt_dict])[summary_header].to_csv(output_path + curdate + 'dpxCheckSummary' + '.csv', index=False)
    return opt_dict


if __name__ == "__main__":
    dpx_check_bydate()